#encoding:utf-8
import os
from gensim import corpora, models, similarities
import numpy as np
from pprint import pprint
import logging
from gensim import corpora
from collections import defaultdict
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer,CountVectorizer

print("config!")
outputPath = "output/"
isDemo = False
if isDemo:
    commonPath = "data/"
    trainPath = commonPath+"train_set_demo.csv"
    testPath = commonPath+"test_set_demo.csv"
    trainNewPath = outputPath+"train_set_demo_new.csv"
    testNewPath = outputPath+"test_set_demo_new.csv"
else:
    commonPath = "../data/"
    trainPath = commonPath+"train_set.csv"
    testPath = commonPath+"test_a.csv"
    trainNewPath = outputPath+"train_set_new.csv"
    testNewPath = outputPath+"test_a_new.csv"

dict_dataframe_path = outputPath+"dict_dataframe.csv"
fileName = 'all'
freq = 1

# 功能 csv 数据 加载
def loadCSVData(fileName):
    df = pd.read_csv(fileName,encoding="utf-8",sep="\t")
    return df

# 数据 特征提取
def text_feature_ex(text,feature_names,minFreq,sep=" "):
    new_text = []
    for word in text.split(sep):
        if word in feature_names:
            new_text.append(word)
    if len(new_text)>=minFreq:
        return sep.join(new_text)
    else:
        return text
print("load data!")
trainDf = loadCSVData(trainPath)
testDf = loadCSVData(testPath)
alldocument = list(trainDf['text'])+list(testDf['text'])
print("TF-idf process!")
tfidfVectorizer = TfidfVectorizer(max_df=0.95,min_df=0.05)
x = tfidfVectorizer.fit_transform(alldocument)  
del alldocument,x
feature_names = tfidfVectorizer.get_feature_names()
del tfidfVectorizer

print("save data!")
trainDf['text_clean'] =  trainDf['text'].apply(text_feature_ex,**{'feature_names':feature_names,'minFreq':1,'sep':" "})
trainDf.columns = ['label','old_text','text']
trainDf[['label','text']].to_csv(trainNewPath,encoding="utf-8",sep="\t",index=None)
del trainDf
testDf['text_clean'] =  testDf['text'].apply(text_feature_ex,**{'feature_names':feature_names,'minFreq':1,'sep':" "})
testDf.columns = ['old_text','text']
testDf[['text']].to_csv(testNewPath,encoding="utf-8",sep="\t",index=None)
del testDf
